﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

namespace WebCrawler
{
    public class HttpCrawler
    {
        private CrawlerOption _option;
        private Queue<PageOption> crawlQueue;
        private Queue<PageOption> downQueue;
        private Uri baseUri;
        public HttpCrawler(CrawlerOption option)
        {
            _option = option;
            Uri uri = new Uri(_option.StartUrl);
            baseUri = new Uri(uri.Scheme + "://"+ uri.Authority);
            crawlQueue = new Queue<PageOption>();
            downQueue = new Queue<PageOption>();
        }
        //string user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36";

        public void Start()
        {
            if (_option == null)
                throw new InvalidOperationException();

            if (!Directory.Exists(_option.TargetPath))
                Directory.CreateDirectory(_option.TargetPath);

            crawlQueue.Enqueue(new PageOption()
            {
                PageUrl = _option.StartUrl,
                PageTitle = string.Empty,
                MatchExpression = _option.SubPage.MatchExpression,
                LinkType = _option.SubPage.LinkType,
                SubPage = _option.SubPage
            });
            Crawel();
            DownloadFiles();
        }

        private void DownloadFiles()
        {
            int len = (int)(Math.Log10(downQueue.Count) + 1);
            //int index = 1;
            PageOption current;
            while(downQueue.TryDequeue(out current))
            {
                Console.WriteLine(string.Format("down file: {0}({1})", current.ParentTitle + current.PageTitle, current.PageUrl));
                DownloadFile(current.PageUrl, /*index.ToString().PadLeft(len, '0') + "_" +*/ current.PageTitle);
                //index++;
            }
        }

        private void DownloadFile(string url, string fileName)
        {
            string savePath = Path.Combine(_option.TargetPath, fileName);
            HttpClient httpClient = new HttpClient();
            var t = httpClient.GetByteArrayAsync(url);
            t.Wait();
            using(Stream responseStream = new MemoryStream(t.Result))
            using(Stream stream = new FileStream(savePath, FileMode.Create))
            {
                byte[] bArr = new byte[1024];
                int size = responseStream.Read(bArr, 0, bArr.Length);
                while (size > 0)
                {
                    stream.Write(bArr, 0, size);
                    size = responseStream.Read(bArr, 0, bArr.Length);
                }
            }
        }

        private void Crawel()
        {
            PageOption current;
            while(crawlQueue.TryDequeue(out current))
            {
                Thread.Sleep(1000);
                string html = HttpGet(current.PageUrl);
                Console.WriteLine(string.Format("fetch html: {0}({1})", current.ParentTitle + current.PageTitle, current.PageUrl));
                if (current.LinkType == LinkTypeEnum.File)
                {
                    Dictionary<string, string> files = FindFiles(html, current.MatchExpression);
                    if (files.Count == 0)
                    {
                        Console.WriteLine(string.Format("no file: {0}({1})", current.ParentTitle + current.PageTitle, current.PageUrl));
                    }
                    foreach(KeyValuePair<string, string> kv in files)
                    downQueue.Enqueue(new PageOption()
                    {
                        PageUrl = new Uri(baseUri, kv.Key).ToString(),
                        PageTitle = kv.Value,
                        LinkType = LinkTypeEnum.File
                    });
                }
                else{
                    Dictionary<string, string> links = GetSubLink(html, current.MatchExpression);
                    foreach (KeyValuePair<string, string> link in links)
                    {
                        crawlQueue.Enqueue(new PageOption()
                        {
                            ParentTitle = current.PageTitle+"/",
                            PageUrl = new Uri(baseUri, link.Key).ToString(),
                            PageTitle = link.Value,
                            MatchExpression = current.SubPage.SubPage.MatchExpression,
                            LinkType = current.SubPage.SubPage.LinkType,
                            SubPage = current.SubPage.SubPage
                        });
                    }
                }

            }
        }

        private Dictionary<string, string> FindFiles(string html, string matchExpression)
        {
            Dictionary<string, string> links = new Dictionary<string, string>();
            Regex reg = new Regex(matchExpression, RegexOptions.IgnoreCase | RegexOptions.Singleline);
            foreach (Match m in reg.Matches(html))
            {
                links.Add(m.Groups["link"].Value, m.Groups["title"].Value.Trim());
            }
            return links;
        }

        private Dictionary<string, string> GetSubLink(string html, string expression)
        {
            Dictionary<string, string> links = new Dictionary<string, string>();
            Regex reg = new Regex(expression, RegexOptions.IgnoreCase | RegexOptions.Singleline);
            foreach(Match m in reg.Matches(html))
            {
                links.Add(m.Groups["link"].Value, m.Groups["title"].Value.Trim());
            }
            return links;
        }

        /// <summary>
        /// 异步GET请求
        /// </summary>
        /// <param name="url"></param>
        /// <param name="headers"></param>
        /// <param name="timeout">请求响应超时时间，单位/s(默认100秒)</param>
        /// <returns></returns>
        public async Task<string> HttpGetAsync(string url, Dictionary<string, string> headers = null, int timeout = 0)
        {
            using (HttpClient client = new HttpClient())
            {
                if (headers != null)
                {
                    foreach (KeyValuePair<string, string> header in headers)
                    {
                        client.DefaultRequestHeaders.Add(header.Key, header.Value);
                    }
                }
                if (timeout > 0)
                {
                    client.Timeout = new TimeSpan(0, 0, timeout);
                }
                Byte[] resultBytes = await client.GetByteArrayAsync(url);
                return Encoding.Default.GetString(resultBytes);
            }
        }

        /// <summary>
        /// 同步GET请求
        /// </summary>
        /// <param name="url"></param>
        /// <param name="headers"></param>
        /// <param name="timeout">请求响应超时时间，单位/s(默认100秒)</param>
        /// <returns></returns>
        public string HttpGet(string url, Dictionary<string, string> headers = null, int timeout = 0)
        {
            using (HttpClient client = new HttpClient())
            {
                if (headers != null)
                {
                    foreach (KeyValuePair<string, string> header in headers)
                    {
                        client.DefaultRequestHeaders.Add(header.Key, header.Value);
                    }
                }
                if (timeout > 0)
                {
                    client.Timeout = new TimeSpan(0, 0, timeout);
                }
                Task<string> task = client.GetStringAsync(url);
                task.Wait();
                if (task.IsCompletedSuccessfully)
                    return task.Result;
                else
                    throw new HttpRequestException(task.AsyncState.ToString());
                //Byte[] resultBytes = client.GetByteArrayAsync(url).Result;
                //return Encoding.UTF8.GetString(resultBytes);
            }
        }

        /// <summary>
        /// 同步POST请求
        /// </summary>
        /// <param name="url"></param>
        /// <param name="postData"></param>
        /// <param name="headers"></param>
        /// <param name="contentType"></param>
        /// <param name="timeout">请求响应超时时间，单位/s(默认100秒)</param>
        /// <param name="encoding">默认UTF8</param>
        /// <returns></returns>
        public string HttpPost(string url, string postData, Dictionary<string, string> headers = null, string contentType = null, int timeout = 0, Encoding encoding = null)
        {
            using (HttpClient client = new HttpClient())
            {
                if (headers != null)
                {
                    foreach (KeyValuePair<string, string> header in headers)
                    {
                        client.DefaultRequestHeaders.Add(header.Key, header.Value);
                    }
                }
                if (timeout > 0)
                {
                    client.Timeout = new TimeSpan(0, 0, timeout);
                }
                using (HttpContent content = new StringContent(postData ?? "", encoding ?? Encoding.UTF8))
                {
                    if (contentType != null)
                    {
                        content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType);
                    }
                    using (HttpResponseMessage responseMessage = client.PostAsync(url, content).Result)
                    {
                        Byte[] resultBytes = responseMessage.Content.ReadAsByteArrayAsync().Result;
                        return Encoding.UTF8.GetString(resultBytes);
                    }
                }
            }
        }

        /// <summary>
        /// 异步POST请求
        /// </summary>
        /// <param name="url"></param>
        /// <param name="postData"></param>
        /// <param name="headers"></param>
        /// <param name="contentType"></param>
        /// <param name="timeout">请求响应超时时间，单位/s(默认100秒)</param>
        /// <param name="encoding">默认UTF8</param>
        /// <returns></returns>
        public async Task<string> HttpPostAsync(string url, string postData, Dictionary<string, string> headers = null, string contentType = null, int timeout = 0, Encoding encoding = null)
        {
            using (HttpClient client = new HttpClient())
            {
                if (headers != null)
                {
                    foreach (KeyValuePair<string, string> header in headers)
                    {
                        client.DefaultRequestHeaders.Add(header.Key, header.Value);
                    }
                }
                if (timeout > 0)
                {
                    client.Timeout = new TimeSpan(0, 0, timeout);
                }
                using (HttpContent content = new StringContent(postData ?? "", encoding ?? Encoding.UTF8))
                {
                    if (contentType != null)
                    {
                        content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType);
                    }
                    using (HttpResponseMessage responseMessage = await client.PostAsync(url, content))
                    {
                        Byte[] resultBytes = await responseMessage.Content.ReadAsByteArrayAsync();
                        return Encoding.UTF8.GetString(resultBytes);
                    }
                }
            }
        }
    }
}
